{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import requests\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Books = pd.DataFrame({'Name': [\"XiYouJi\", \"HongLouMeng\", \"ShuiHuZhuan\", \"SanGuoYanYi\"], \n",
    "                     'url':  [\"https://xiyouji.5000yan.com/\", \"https://hongloumeng.5000yan.com/\", \"https://shuihu.5000yan.com/\", \"https://sanguo.5000yan.com/\"]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 爬取西游记、红楼梦、水浒传、三国演义\n",
    "\n",
    "for idx, row in Books.iterrows():\n",
    "    book_name = row['Name']\n",
    "    url = row['url']\n",
    "    response = requests.get(url)\n",
    "    soup = bs(response.content)\n",
    "    contents = soup.find('div',class_='p-2 my-2 bg-white rounded')\n",
    "    if contents is None:\n",
    "        contents = soup.find('div',class_='p-3 my-2 bg-white rounded')\n",
    "    contents = contents.ul.children\n",
    "    \n",
    "    for chapter in contents:\n",
    "        if chapter=='\\n' or chapter is None:\n",
    "            continue\n",
    "        # print(chapter)\n",
    "        chapter_name = chapter.a.text.replace('?','')\n",
    "        chapter_link = chapter.a.get('href')\n",
    "        # 获取章节内容\n",
    "        chapter_response = requests.get(chapter_link)\n",
    "        chapter_soup = bs(chapter_response.content)\n",
    "        chapter_content = chapter_soup.find('div',class_='py-2 px-2 px-md-4 bg-white divider')\n",
    "        # 红楼梦有几章格式有问题抓不到content这里单独处理\n",
    "        if chapter_content is None:\n",
    "            chapter_content = str(chapter_soup.find('div',class_='py-2 px-3 my-2 bg-white rounded text-center'))+str(chapter_soup.find('div',class_='py-3 px-3 my-2 bg-white rounded'))\n",
    "            chapter_content = bs(chapter_content,'html.parser')\n",
    "        # 获取文本，并去除空行\n",
    "        if chapter_content is None or chapter_content=='\\n':\n",
    "            continue\n",
    "        chapter_text = chapter_content.text\n",
    "        lines = chapter_text.splitlines()\n",
    "        lines = [line for line in lines if line.strip()]# 移除字符串前后的指定字符(空格或换行)\n",
    "        chapter_text = '\\n'.join(lines)\n",
    "        # print(chapter_text)\n",
    "\n",
    "        # 文本保存\n",
    "        if not os.path.exists(\".\\\\data\\\\\"+book_name):\n",
    "            os.makedirs(\".\\\\data\\\\\"+book_name)\n",
    "        with open(\".\\\\data\\\\\"+book_name+\"\\\\\"+chapter_name+\".txt\", 'w', encoding='utf-8') as handler:\n",
    "            handler.write(chapter_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
